package com.ifreeee.practice.pc.famousWorks;

import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.ifreeee.practice.pc.famousWorks.exclude.HlmExclude;
import com.ifreeee.practice.pc.famousWorks.exclude.JpmExclude;
import jdk.nashorn.internal.runtime.ParserException;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.net.URL;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * 小马哥名著 爬虫
 */
public class FamousWorks {

    private static final Integer BATCH_NUM = 3;

    private static final ThreadPoolExecutor poolExecutor
            = new ThreadPoolExecutor(BATCH_NUM, BATCH_NUM
            , 1, TimeUnit.MINUTES
            , new LinkedBlockingQueue<>(3000)
            , new ThreadPoolExecutor.CallerRunsPolicy());

    /**
     * 文件下载到本地的哪个路径下
     */
    private static final String LOCAL_PATH = "D:\\名著";

    /**
     * 音频的文件后缀
     */
    private static final String LOCAL_PATH_SUFFIX = ".mp3";

    /**
     * 初始序号
     */
    private static final Integer INIT_NO = 1;

    /**
     * 序号与标题的分隔符
     */
    private static final String NO_TITLE_SEP = "-";

    /**
     * 本地文件分隔符
     */
    private static final String SEP = File.separator;

    /**
     * html 获取 入口 id
     */
    private static final String CONTENT_ID = "js_content";

    /**
     * 字符集
     */
    private static final String CHARSET = "utf-8";

    /**
     * 音频链接前缀
     */
    private static final String VOICE_URL_PRE = "https://res.wx.qq.com/voice/getvoice?mediaid=";

    /**
     * 初始重试次数
     */
    private static final Integer INIT_RE_TRY = 3;

    /**
     * 名著名称 与 页面路径的对应关系
     */
    private static Multimap<String, String> nameToUrlMap = LinkedHashMultimap.<String, String>create();

    /**
     * 排除
     */
    private static Map<String, Set<String>> excludeNameToTitle = new HashMap<>();

    /**
     * 获取标签
     */
    private static List<String> tagList = new ArrayList<>();

    static {
        //nameToUrlMap.put("红楼梦", "https://mp.weixin.qq.com/s/d058Ugbek3DTZlkYIwen-A");
        //nameToUrlMap.put("红楼梦", "https://mp.weixin.qq.com/s/yoR_o2I73X_KaLu06k0UUA");
        //nameToUrlMap.put("金瓶梅", "https://mp.weixin.qq.com/s/UaR5p83o3rUX1V2u9KImMA");
        //nameToUrlMap.put("金瓶梅", "https://mp.weixin.qq.com/s/9tw7HrscY1sPS9BRE2f6WQ");

        nameToUrlMap.put("红楼梦", "https://mp.weixin.qq.com/s/CGd92xFEZpv5BRXGLzHYTA");
        //nameToUrlMap.put("金瓶梅", "https://mp.weixin.qq.com/s/zwsrWSMf16FxTZql_lCZLA");
        //nameToUrlMap.put("金瓶梅", "https://mp.weixin.qq.com/s/s52vcnYAelfqSrxrxYPCRA");

        excludeNameToTitle.put("金瓶梅", JpmExclude.excludeSet());
        excludeNameToTitle.put("红楼梦", HlmExclude.excludeSet());

        tagList.add("mpvoice");
        tagList.add("mp-common-mpaudio");
    }

    /**
     * 名著爬虫
     *
     * @param args
     */
    public static void main(String[] args) {
        download();
    }

    private static void download() {
        Set<String> mingzhuNameSet = nameToUrlMap.keySet();
        Map<String, Map<String, String>> nameToTitleToUrlMap = new HashMap<>();
        for (String name : mingzhuNameSet) {
            // 目录的 url
            Collection<String> catalogueUrlCollect = nameToUrlMap.get(name);
            Map<String, String> allTitleToUrl = new LinkedHashMap<>();
            for (String catalogueUrl : catalogueUrlCollect) {
                //String catalogueHtml = HttpUtil.sendGet(catalogueUrl, CHARSET);
                String catalogueHtml = HttpRequest.sendGet(catalogueUrl, null/*CHARSET*/);
                Map<String, String> titleToUrl = getTitleToUrl(catalogueHtml);
                if (MapUtils.isEmpty(titleToUrl)) {
                    continue;
                }
                allTitleToUrl.putAll(titleToUrl);
            }
            nameToTitleToUrlMap.put(name, allTitleToUrl);
        }
        printMap(nameToTitleToUrlMap);
        List<UrlWrapper> urlWrapperList = resolvePath(nameToTitleToUrlMap);
        downloadFile(urlWrapperList);
        // 关闭线程池, 否则程序执行完不退出
        poolExecutor.shutdown();
        // mp3 的 title 处理
        Mp3Title mp3Title = new Mp3Title(LOCAL_PATH);
        mp3Title.setTitle();
    }

    private static Map<String, String> getTitleToUrl(String catalogueHtml) {
        Map<String, String> resMap = new LinkedHashMap<>();
        HQJSoup hqjSoup = null;
        try {
            hqjSoup = new HQJSoup(catalogueHtml);
        } catch (ParserException e) {
            e.printStackTrace();
        }
        if (hqjSoup == null) {
            return resMap;
        }
        Element jsContent = hqjSoup.getElementById(CONTENT_ID);
        if (jsContent == null) {
            return resMap;
        }
        Elements ps = jsContent.getAllElements();
        if (CollectionUtils.isEmpty(ps)) {
            return resMap;
        }
        Elements aEles = new Elements();
        for (Element element : ps) {
            Elements as = element.getElementsByTag("a");
            if (CollectionUtils.isEmpty(as)) {
                continue;
            }
            for (Element a : as) {
                if (a == null) {
                    continue;
                }
                aEles.add(a);
            }
        }
        if (CollectionUtils.isEmpty(aEles)) {
            return resMap;
        }

        for (Element aEle : aEles) {
            String aTagTitle = getATagTitle(aEle);
            String aTagUrl = getATagUrl(aEle);
            if (StringUtils.isBlank(aTagTitle)
                    || StringUtils.isBlank(aTagUrl)) {
                continue;
            }
            resMap.remove(aTagTitle);
            resMap.put(aTagTitle, aTagUrl);
        }
        return resMap;
    }

    /**
     * 获取 a 标签的标题
     *
     * @param aEle
     * @return
     */
    private static String getATagTitle(Element aEle) {
        String resStr = "";
        if (aEle == null) {
            return resStr;
        }
        if (aEle.hasText()) {
            return aEle.text();
        }
        return resStr;
    }

    /**
     * 获取 a 标签的标题
     * @param aEle
     * @return
     */
    private static String getATagUrl(Element aEle) {
        String resStr = "";
        if (aEle == null) {
            return resStr;
        }
        if (aEle.hasAttr("href")) {
            return aEle.attr("href");
        }
        return resStr;
    }

    private static void printMap(Map map) {
        System.out.println(printMap("", map));
    }

    /**
     * 按行打印 map
     * @param map
     * @return
     */
    private static StringBuilder printMap(String pre, Map map) {
        StringBuilder sb = new StringBuilder();
        if (MapUtils.isEmpty(map)) {
            return sb;
        }
        map.forEach((k, v) -> {
            sb.append(pre);
            if (v instanceof Map) {
                sb.append(k);
                sb.append(": ");
                sb.append("\n");
                sb.append(printMap(pre + "    ", (Map) v));
            } else {
                sb.append(k);
                sb.append("    <====>    ");
                sb.append(v);
            }
            sb.append("\n");
        });
        return sb;
    }

    private static List<UrlWrapper> resolvePath(Map<String, Map<String, String>> nameToTitleToUrlMap) {
        if (MapUtils.isEmpty(nameToTitleToUrlMap)) {
            return Collections.emptyList();
        }
        StringBuilder sb = new StringBuilder();

        List<UrlWrapper> urlWrapperList = new ArrayList<>();
        // 解析本地保存路径
        nameToTitleToUrlMap.forEach((name, m1) -> {
            if (MapUtils.isEmpty(m1)) {
                return;
            }
            SerialNoGen serialNoGen = new SerialNoGen(INIT_NO);
            m1.forEach((title, url) -> {
                Set<String> excludeSet = excludeNameToTitle.get(name);
                if (excludeSet != null && excludeSet.contains(title)) {
                    return;
                }
                String key = null;
                sb.delete(0, sb.length());
                sb.append(LOCAL_PATH);
                if (!StringUtils.endsWith(LOCAL_PATH, SEP)) {
                    sb.append(SEP);
                }
                sb.append(name);
                sb.append(SEP);
                sb.append(serialNoGen.get());
                sb.append(NO_TITLE_SEP);
                sb.append(title);
                sb.append(LOCAL_PATH_SUFFIX);
                key = sb.toString();
                sb.delete(0, sb.length());
                // TODO
                urlWrapperList.add(new UrlWrapper(key
                        , url.replaceFirst("http://mp.weixin.qq.com/", "https://mp.weixin.qq.com/")));
            });
        });

        if (CollectionUtils.isEmpty(urlWrapperList)) {
            return Collections.emptyList();
        }

        // 解析音频路径
        StringBuffer errMes = new StringBuffer();
        List<List<UrlWrapper>> partitionUrlWrappers = Lists.partition(urlWrapperList, urlWrapperList.size() / BATCH_NUM + 1);
        CountDownLatch countDownLatch = new CountDownLatch(partitionUrlWrappers.size());
        for (List<UrlWrapper> partitionUrlWrapper : partitionUrlWrappers) {
            poolExecutor.execute(() -> {
                if (CollectionUtils.isEmpty(partitionUrlWrapper)) {
                    countDownLatch.countDown();
                    return;
                }
                for (UrlWrapper urlWrapper : partitionUrlWrapper) {
                    String localUrl = urlWrapper.getLocalUrl();
                    String voiceUrl = urlWrapper.getVoiceUrl();
                    String realVoiceUrl = resolveAudioPath(voiceUrl, INIT_RE_TRY);
                    if (StringUtils.isBlank(realVoiceUrl)) {
                        errMes.append("未获取成功: " + localUrl + "====" + voiceUrl + "\n");
                        continue;
                    }
                    urlWrapper.setVoiceUrl(realVoiceUrl);
                }
                countDownLatch.countDown();
            });
        }
        try {
            countDownLatch.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        System.out.println("志超解析错误信息: \n" + errMes);
        return urlWrapperList;
    }

    /**
     * 解析音频路径
     * 需要并发执行, 否则很慢
     * @return
     */
    private static String resolveAudioPath(String audioUrl, int reTry) {
        String resStr = "";
        String audioHtml = HttpRequest.sendGet(audioUrl, null);
        HQJSoup hqjSoup = null;
        try {
            hqjSoup = new HQJSoup(audioHtml);
        } catch (ParserException e) {
            e.printStackTrace();
        }
        for (String tag : tagList) {
            if (StringUtils.isNotBlank(resStr))
                break;
            resStr = Optional.ofNullable(hqjSoup)
                    .map(h -> h.getElementsByTagName(tag))
                    .map(m -> m.attr("voice_encode_fileid"))
                    .orElse(resStr);
        }
        if (StringUtils.isBlank(resStr)) {
            if (reTry == 0) {
                return resStr;
            }
            return resolveAudioPath(audioUrl, reTry - 1);
        }
        return VOICE_URL_PRE + resStr;
    }

    public static void downloadFile(List<UrlWrapper> urlWrapperList) {
        if (CollectionUtils.isEmpty(urlWrapperList)) {
            return;
        }
        // 解析音频路径
        StringBuffer errMes = new StringBuffer();
        List<List<UrlWrapper>> partitionUrlWrappers = Lists.partition(urlWrapperList, urlWrapperList.size() / BATCH_NUM + 1);
        CountDownLatch countDownLatch = new CountDownLatch(partitionUrlWrappers.size());
        for (List<UrlWrapper> partitionUrlWrapper : partitionUrlWrappers) {
            poolExecutor.execute(() -> {
                if (CollectionUtils.isEmpty(partitionUrlWrapper)) {
                    countDownLatch.countDown();
                    return;
                }
                for (UrlWrapper urlWrapper : partitionUrlWrapper) {
                    String urlStr = urlWrapper.getVoiceUrl();
                    URL url = null;
                    try {
                        url = new URL(urlStr);
                        String tempFileName = urlWrapper.getLocalUrl();
                        File temp = new File(tempFileName);
                        FileUtils.copyURLToFile(url, temp);
                    } catch (Exception e) {
                        errMes.append("下载异常: " + urlWrapper.toString() + "\n");
                    }
                }
                countDownLatch.countDown();
            });
        }
        try {
            countDownLatch.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        System.out.println("志超下载错误信息: \n" + errMes);
    }

}
